#!/bin/bash
export PYSPARK_PYTHON=python3.7
export PATH=${PATH}
export PYSPARK_DRIVER_PYTHON=python3.7

if [ -d "$PWD/dist" ]; then
    rm -f $PWD/dist/*
else
    mkdir -p $PWD/dist
fi
rm -rf userprofile.egg-info
#python3.7 setup.py bdist_egg > /dev/null
python3.7 setup.py bdist_egg 
egg=$PWD/dist/`ls $PWD/dist/ `
echo $egg

time_start=$(date +%s)

spark-submit  --num-executors=5 \
              --executor-memory=10G \
              --executor-cores=10 \
              --driver-memory=4G \
              --conf spark.dynamicAllocation.maxExecutors=10 \
              --conf spark.default.parallelism=150 \
              --conf spark.memory.fraction=0.85 \
              --conf spark.memory.storagefraction=0.3 \
              --conf spark.memory.offHeap.size=2G \
              --conf spark.executor.memoryOverhead=1024 \
              --conf spark.core.connection.ack.wait.timeout=300 \
              --conf 'spark.driver.extraJavaOptions=-Dlog4j.configuration=file:./env/conf/log4j.properties'  \
              --jars ./env/bulid/mysql-connector-java-8.0.16.jar  \
              --driver-class-path ./env/bulid/mysql-connector-java-8.0.16.jar \
              --py-files $egg \
              --files /opt/hive/conf/hive-site.xml \
              ./server/main.py

time_end=$(date +%s)
echo $(( (time_end - time_start )/60 ))
